package com.huawei.spider.cnki.plugins;

import com.huawei.spider.cnki.common.CnkiConstant;
import com.huawei.spider.cnki.common.CrawlConfig;
import com.huawei.spider.cnki.config.GetConfig;
import com.huawei.spider.cnki.util.FileUtils;
import com.huawei.spider.cnki.util.UrlSafeEncoder;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.client.CookieStore;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.huawei.spider.cnki.util.ExcelUtil.createBody;
import static com.huawei.spider.cnki.util.ExcelUtil.createHeader;

/**
 * @Author: liminghui
 * @Date: 2020/5/17 9:53
 * @Version:
 * @Description: 构建搜索类  实现搜索方法
 */
public class SearchTools {
    public static String change_page_url;
    public static String download_url;
    public static String get_result_url;
    public static CloseableHttpClient httpClient = null;
    public static CookieStore cookieStore = null;
    public static HttpClientContext context = null;
    public static BasicClientCookie cookie = null;
    public static String cnkiUserKey = set_new_guid();
    public static Integer cur_page_num = 1;
    public static List<List<String>> dataList = new ArrayList<>();

    static {
        // 创建cookie store的本地实例
        cookieStore = new BasicCookieStore();
        cookie = new BasicClientCookie("cnkiUserKey", cnkiUserKey);
        cookieStore.addCookie(cookie);
        // 创建HttpClient上下文
        context = HttpClientContext.create();
        context.setCookieStore(cookieStore);
    }

    /**
     * 删除某个文件夹下所有文件
     *
     * @param dataSrc
     */
    public void deleteData(String dataSrc) {
        //当前项目下路径[本地和Jar外文件夹]
        File file1 = new File("");
        String path = null;
        try {
            path = file1.getCanonicalPath();
        } catch (IOException e) {
            System.out.println("未找到项目路径: IOException---->" + e);
        }
        dataSrc = path + "/data";
        List<File> fileList = null;
        Integer fileSize = 0;
        try {
            fileList = FileUtils.getFileList(new File(dataSrc));
            fileSize = fileList.size();
        } catch (IOException e) {
            System.out.println("没找到文件 Exception:" + e);
        }
        for (File file : fileList) {
            file.delete();
        }
        System.out.println("成功清除" + dataSrc + "下共" + fileSize + "个文件!");
    }

    /**
     * 第一次发送post请求
     * 再一次发送get请求,这次请求没有写文献等东西
     * 两次请求来获得文献列表
     *
     * @param ueser_input
     */
    public void search_reference(Map<String, String> ueser_input) throws IOException {
        Map<String, String> static_post_data = new HashMap<>();
        static_post_data.put("action", "");
        static_post_data.put("NaviCode", "*");
        static_post_data.put("ua", "1.21");
        static_post_data.put("isinEn", "1");
        static_post_data.put("PageName", "ASP.brief_default_result_aspx");
        static_post_data.put("DbPrefix", "SCDB");
        static_post_data.put("DbCatalog", "中国学术期刊网络出版总库");
        static_post_data.put("ConfigFile", "CJFQ.xml");
        static_post_data.put("db_opt", "CJFQ,CDFD,CMFD,CPFD,IPFD,CCND,CCJD"); // 搜索类别（CNKI右侧")
        static_post_data.put("db_value", "中国学术期刊网络出版总库");
        static_post_data.put("year_type", "echar");
        static_post_data.put("his", "0");
        static_post_data.put("db_cjfqview", "中国学术期刊网络出版总库,WWJD");
        static_post_data.put("db_cflqview", "中国学术期刊网络出版总库");
        SimpleDateFormat sdf = new SimpleDateFormat("EEE MMM dd HH:mm:ss yyyy", Locale.ENGLISH);
        String format = sdf.format(new Date());
        static_post_data.put("__", format + " GMT+0800 (中国标准时间)");
        //  将固定字段与自定义字段组合
        Map<String, String> post_data = new HashMap<>();
        post_data.putAll(static_post_data);
        post_data.putAll(ueser_input);
        Map<String, String> headers = GetConfig.getHeaders();

        httpClient = HttpClients.createDefault();
        // 1.必须有第一次请求，否则会提示服务器没有用户
        String first_post_res = doPost(httpClient, CnkiConstant.SEARCH_HANDLE_URL, headers, post_data);

        // 2.get请求中需要传入第一个检索条件的值
        String key_value = URLEncoder.encode(ueser_input.get("txt_1_value1"), "UTF-8");
        get_result_url = CnkiConstant.GET_PAGE_URL + first_post_res + "&t=1544249384932&keyValue=" + key_value + "&S=1&sorttype=";

        // 检索结果的第一个页面
        String second_get_res = doGet(httpClient, get_result_url, headers);

        //3.解析页面url
        Document document = Jsoup.parse(second_get_res);
        Elements elements = document.select("#ctl00 > table > tbody > tr:nth-child(3) > td > table > tbody > tr > td > div");
        //根据td标签来划分
        Element element = elements.get(0);
        Elements elm2 = element.getElementsByTag("a");
        //?curpage=2&RecordsPerPage=20&QueryID=0&ID=&turnpage=1&tpagemode=L&dbPrefix=SCDB&Fields=&DisplayMode=listmode&PageName=ASP.brief_default_result_aspx&isinEn=1&
        change_page_url = elm2.get(0).attr("href");

        Integer page = pre_parse_page(second_get_res);
        parse_page(page, second_get_res);
        httpClient.close();
    }

    public void writeToExcel() throws IOException {
        // String path = System.getProperty("user.dir");
        //当前项目下路径
        File file = new File("");
        String path = file.getCanonicalPath();
        String dirPath = path + "/data";
        File file1 = new File(dirPath);
        if (!file1.exists()) {
            file1.mkdirs();
        }
        String filePath = dirPath + "/知网下载汇总表.xls";
        OutputStream os = new FileOutputStream(filePath, true);
        String[] headers = {"序号", "题名", "作者", "来源", "发表时间", "数据库", "被引", "下载", "阅读", "收藏", "详情链接"};
        HSSFWorkbook workBook = new HSSFWorkbook();
        HSSFSheet sheet = workBook.createSheet("表1");
        createHeader(headers, sheet);
        createBody(dataList, sheet);
        workBook.write(os);
        System.out.println("数据已经下载到Excel:" + filePath);
        os.close();
    }

    /**
     * 保存页面信息
     * 解析每一页的下载地址
     *
     * @param download_page_left
     * @param page_source
     */
    public void parse_page(int download_page_left, String page_source) throws IOException {
        Document document = Jsoup.parse(page_source);
        Element tr_table = document.select("#ctl00 > table > tbody > tr:nth-child(2) > td > table").first();
        Elements trs = null;
        try {
            //找到table标签
            trs = tr_table.getElementsByTag("tr");
            trs.remove(0);
            for (int i = 0; i < trs.size(); i++) {
                Element tr_element = trs.get(i);
                Elements tds = tr_element.getElementsByTag("td");
                for (Element td : tds) {
                    //System.out.println(td.text());
                }
            }
        } catch (Exception e) {
            System.out.println("出现验证码 Exception:" + e);
            // TODO 处理验证码
            parse_page(download_page_left, page_source);
            return;
        }

        // 遍历每一行
        for (Element tr : trs) {
            String tr_text = null;
            String download_url = null;
            String detail_url = null;
            StringBuilder sb = new StringBuilder();
            // 遍历每一列
            Elements tds = tr.select("td");
            for (Element td : tds) {
                //# 因为一列中的信息非常杂乱，此处进行二次拼接
                String td_text = td.text();
                if (td_text.contains(" ")) {
                    td_text = td_text.split(" ")[0];
                }
                sb.append(td_text).append(" ");
            }
            tr_text = sb.deleteCharAt(sb.length() - 1).toString();
            // 寻找下载链接
            Elements aChooseds = tr.getElementsByTag("a");
            for (Element aChoosed : aChooseds) {
                String className = aChoosed.className();
                if ("briefDl_D".equals(className)) {
                    detail_url = aChoosed.attr("href");
                    detail_url = detail_url.substring(2);
                    break;
                }
            }

            //寻找详情链接
            for (Element aChoosed : aChooseds) {
                String className = aChoosed.className();
                if ("fz14".equals(className)) {
                    download_url = aChoosed.attr("href");
                    break;
                }
            }

            //将每一篇文献的信息分组
            String[] single_refence_list = tr_text.split(" ");
            download_refence(download_url, single_refence_list);
            //将每一篇的信息写入excel
            List<String> data = new ArrayList<>();//存放数据体 一条数据
            for (String field : single_refence_list) {
                data.add(field);
            }
            data.add(download_url);

            //  是否开启详情页数据抓取
            if (CrawlConfig.isDetailPage == 1) {
                try {
                    Thread.sleep(CrawlConfig.stepWaitTime);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
                //get_detail_page(get_result_url, detail_url, single_refence_list, download_url);
            }
            dataList.add(data);
        }

        //download_page_left为剩余等待遍历页面
        if (download_page_left > 1) {
            cur_page_num += 1;
            get_another_page(download_page_left);
        }

    }

    /**
     * 请求其他页面和请求第一个页面形式不同
     * 重新构造请求
     *
     * @param download_page_left
     */
    public void get_another_page(int download_page_left) throws IOException {
        get_result_url = CnkiConstant.CHANGE_PAGE_URL + change_page_url.replaceAll(".*?curpage=(\\d+).*?", "?curpage=" + cur_page_num);
        String get_res = doGet(httpClient, get_result_url, GetConfig.getHeaders());
        download_page_left--;
        parse_page(download_page_left, get_res);
    }

    /**
     * 拼接下载地址
     * 进行文献下载
     *
     * @param url
     * @param single_refence_list
     */
    public void download_refence(String url, String[] single_refence_list) throws IOException {
        System.out.println("正在下载: " + single_refence_list[1] + ".caj");
        String name = single_refence_list[1] + '_' + single_refence_list[2];
        //检查文件命名，防止网站资源有特殊字符本地无法保存
        name = name.replaceAll("[\\/\\*\\?\"<>\\|:;]", "");
        download_url = CnkiConstant.DOWNLOAD_URL + url;
        //保存下载链接
        if (CrawlConfig.isDownLoadLink == 1) {
            try {
                BufferedWriter br = new BufferedWriter(new FileWriter("data/Links.txt", true));
                br.write(download_url);
                br.newLine();
                br.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        //检查是否开启下载模式
        if (CrawlConfig.isCrackCode == 1) {
            File file = new File("data/CAJs");
            if (!file.exists()) {
                file.mkdirs();
            }
            String refence_file = doGet(httpClient, download_url, GetConfig.getHeaders());
            try {
                String destPath = "data/CAJs/" + name + ".caj";
                BufferedWriter br = new BufferedWriter(new FileWriter(destPath));
                br.write(refence_file);
                br.close();
                Thread.sleep(CrawlConfig.stepWaitTime);
            } catch (IOException e) {
                e.printStackTrace();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }

    /**
     * 用户选择需要检索的页数
     *
     * @param page_source
     * @return
     */
    public Integer pre_parse_page(String page_source) {
        Pattern reference_num_pattern_compile = Pattern.compile(".*?找到&nbsp;(.*?)&nbsp;");
        Matcher matcher = reference_num_pattern_compile.matcher(page_source);
        int reference_num = 0;
        if (matcher.find()) {
            String reference_num_str = matcher.group(1);
            reference_num = Integer.parseInt(reference_num_str.replace(",", ""));
        }
        System.out.println("检索到" + reference_num + "条结果，全部下载大约需要" +
                s2h(reference_num * 5) + "。");
        System.out.println("是否要全部下载（y/n）?");
        Scanner scanner = new Scanner(System.in);
        String is_all_download = scanner.nextLine();
        // # 将所有数量根据每页20计算多少页
        Integer page = 0;
        if ("y".equalsIgnoreCase(is_all_download)) {
            page = reference_num % 20 == 0 ? reference_num / 20 : reference_num / 20 + 1;
        } else {
            System.out.println("请输入需要下载的数量：");
            Integer select_download_num = scanner.nextInt();
            while (true) {
                if (select_download_num > reference_num) {
                    System.out.println("输入数量大于检索结果，请重新输入！");
                    System.out.println("请输入需要下载的数量（不满一页将下载整页）：");
                    select_download_num = scanner.nextInt();
                } else {
                    // # 不满一页的下载一整页
                    page = select_download_num % 20 == 0 ? select_download_num / 20 : select_download_num / 20 + 1;
                    System.out.println("开始下载前" + page + "页所有文件，预计用时" + s2h(page * 20 * 5));
                    System.out.println("－－－－－－－－－－－－－－－－－－－－－－－－－－");
                    return page;
                }
            }
        }

        return page;
    }

    /**
     * 将秒数--->时分秒
     *
     * @param seconds
     * @return
     */
    public String s2h(int seconds) {
        if (seconds < 60) {
            return seconds + "秒";
        } else if (seconds > 60 && seconds < 3600) {
            int m = seconds / 60;
            int s = seconds % 60;
            return m + "分" + s + "秒";
        } else {
            int h = seconds / 3600;
            int m = (seconds % 3600) / 60;
            int s = (seconds % 3600) % 60;
            return h + "小时" + m + "分钟" + s + "秒";
        }

    }


    public String doGet(CloseableHttpClient httpClient, String url, Map<String, String> headerMap) throws IOException {
        String htmlStr = "";
        // 1) 确定首页url
        // 2) 创建httpClient对象
        // 3) 创建请求方式的对象 : get  post    相当于 request
        HttpGet httpGet = new HttpGet(url);
        // 4) 设置请求参数, 请求头
        // setHeader()  和 addHeader()  : 都是用来设置请求头, set是用来设置一个头对应一个value,add设置一个头对应多个value的情况
        for (Map.Entry<String, String> entry : headerMap.entrySet()) {
            httpGet.setHeader(entry.getKey(), entry.getValue());
        }
        // 5) 发送请求, 获取响应对象:response
        CloseableHttpResponse response = httpClient.execute(httpGet);
        // 6) 获取数据:
        // 6.1) 获取响应行
        int code = response.getStatusLine().getStatusCode();  // 获取状态码
        System.out.println("get请求状态码:" + code);
        System.out.println("url:" + url);
        if (200 == code) {
            // 请求成功
            // 6.2) 获取响应头
            Header[] headers = response.getHeaders("Content-Type");
            for (Header header : headers) {
                //System.out.println(header.getName() + ":" + header.getValue());
            }
            // 6.3) 获取响应体
            HttpEntity httpEntity = response.getEntity();
            //InputStream inputStream = httpEntity.getContent();  // 这种方式一般是用来获取图片, 视频  二进制的数据
            htmlStr = EntityUtils.toString(httpEntity, "UTF-8");
            //System.out.println(htmlStr);
        }
        return htmlStr;
    }

    public String doPost(CloseableHttpClient httpClient, String url, Map<String, String> headerMap, Map<String, String> post_data) throws IOException {
        String htmlStr = "";
        // 1) 确定首页url

        // 2) 创建httpClient对象

        // 3) 创建请求方式的对象   ---- request
        HttpPost httpPost = new HttpPost(url);
        // 3.1配置请求参数实例
        RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(35000)// 设置连接主机服务超时时间
                .setConnectionRequestTimeout(35000)// 设置连接请求超时时间
                .setSocketTimeout(60000)// 设置读取数据连接超时时间
                .build();
        // 为httpPost实例设置配置
        httpPost.setConfig(requestConfig);

        // 4) 设置请求参数, 请求头和表单参数
        //4.1请求头参数
        if (headerMap != null) {
            for (Map.Entry<String, String> entry : headerMap.entrySet()) {
                httpPost.addHeader(entry.getKey(), entry.getValue());
            }
        }
        //4.2表单参数
        List<BasicNameValuePair> pairList = new ArrayList<BasicNameValuePair>();
        if (post_data != null) {
            for (Map.Entry<String, String> entry : post_data.entrySet()) {
                pairList.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
            }
        }
        HttpEntity entity = new UrlEncodedFormEntity(pairList, "UTF-8");
        httpPost.setEntity(entity);

        // 5) 发送请求, 获取响应对象:response
        CloseableHttpResponse response = httpClient.execute(httpPost);

        // 6) 获取数据:
        // 6.1) 获取响应行
        int code = response.getStatusLine().getStatusCode();
        System.out.println("post请求状态码:" + code);
        System.out.println("url:" + url);
        if (200 == code) {
            // 6.2) 获取响应头
            Header[] headers = response.getAllHeaders();
            for (Header header : headers) {
                // System.out.println(header.getName() + ":" + header.getValue());
            }

            // 6.3) 获取响应体
            HttpEntity httpEntity = response.getEntity();
            htmlStr = EntityUtils.toString(httpEntity, "UTF-8");
            //System.out.println(htmlStr);
        }

        return htmlStr;
    }


    /**
     * 发送三次请求
     * 前两次服务器注册 最后一次正式跳转
     *
     * @param referer
     * @param detail_url
     * @param single_refence_list
     * @param download_url
     */
    public void get_detail_page(String referer, String detail_url, String[] single_refence_list, String download_url) throws IOException {
        Map<String, String> headers = GetConfig.getHeaders();
        headers.put("Referer", referer);
        Pattern p = Pattern.compile(".*?FileName=(.*?)&.*?DbCode=(.*?)&");
        Matcher matcher = p.matcher(download_url);
        String fileName = "";
        String dbCode = "";
        if (matcher.find()) {
            fileName = matcher.group(1);
            dbCode = matcher.group(2);
        }

        //前两次请求需要的验证参数
        Map<String, String> params = new HashMap<>();
        params.put("curUrl", "detail.aspx?dbCode=" + dbCode + "&fileName=" + fileName);
        params.put("referUrl", referer + "#J_ORDER&");
        params.put("cnkiUserKey", cnkiUserKey);
        params.put("action", "file");
        params.put("userName", "");
        params.put("td", "1544605318654");

        // TODO 因需要登录暂停开发
       /* // 先向服务器发送两次请求  get
        String firstRequestUrl = "http://i.shufang.cnki.net/KRS/KRSWriteHandler.ashx";
        String firstRequestUrl1 = assemblyGetUrl(firstRequestUrl, params);
        String secondRequestUrl = "http://kns.cnki.net/KRS/KRSWriteHandler.ashx";
        String secondRequestUrl2 = assemblyGetUrl(secondRequestUrl, params);
        String firstRequestResult = doGet(httpClient, firstRequestUrl1, headers);
        String secondRequestResult = doGet(httpClient, secondRequestUrl2, headers);

        // 第三次
        detail_url = "http://kns.cnki.net" + detail_url;
        String thirdResult = doGet(httpClient, detail_url, headers);
        System.out.println(thirdResult);
        // 保存内容到excel
        // self.excel.save('data/Reference_detail.xls')*/


    }


    // 生成秘钥
    public static String set_new_guid() {
        String guid = "";
        for (int i = 1; i < 32; i++) {
            String n = Integer.toHexString((int) Math.floor(Math.random() * 16.0));
            guid += n;
            if (i == 8 || i == 12 || i == 16 || i == 20) {
                guid += "-";
            }
        }
        return guid;
    }

    //组装getUrl
    public String assemblyGetUrl(String url, Map<String, String> params) {
        StringBuilder sb = null;
        try {
            sb = new StringBuilder();
            sb.append(url).append("?");
            for (Map.Entry<String, String> entry : params.entrySet()) {
                String getValue = UrlSafeEncoder.encode(entry.getValue(), "UTF-8", '/');
                sb.append(entry.getKey()).append("=").append(getValue).append("&");
            }
            sb.deleteCharAt(sb.length() - 1);
        } catch (UnsupportedEncodingException e) {
            System.out.println("failed to assemblyGetUrl. UnsupportedEncodingException:" + e);
        }
        return sb.toString();
    }


}
